{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Demo - Performance Comparison (CIFAR10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Load CIAFR10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "# https://github.com/RobustBench/robustbench\n",
    "from robustbench.data import load_cifar10\n",
    "from robustbench.utils import load_model, clean_accuracy\n",
    "from utils import l2_distance\n",
    "\n",
    "images, labels = load_cifar10(n_examples=50)\n",
    "device = \"cuda\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Standard Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: Wong2020Fast\n",
      "- Standard Acc: 0.92\n",
      "Model: Rice2020Overfitting\n",
      "- Standard Acc: 0.86\n",
      "Model: Carmon2019Unlabeled\n",
      "- Standard Acc: 0.92\n"
     ]
    }
   ],
   "source": [
    "model_list = ['Wong2020Fast', 'Rice2020Overfitting', 'Carmon2019Unlabeled']\n",
    "for model_name in model_list:\n",
    "    model = load_model(model_name, norm='Linf').to(device)\n",
    "    acc = clean_accuracy(model, images.to(device), labels.to(device))\n",
    "    print('Model: {}'.format(model_name))\n",
    "    print('- Standard Acc: {}'.format(acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Torchattacks, Foolbox and ART"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torchattacks 2.13.3\n",
      "foolbox 3.0.0\n",
      "art 1.2.0\n"
     ]
    }
   ],
   "source": [
    "import datetime\n",
    "import numpy as np\n",
    "import warnings\n",
    "\n",
    "warnings.filterwarnings(action='ignore')\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "\n",
    "# https://github.com/Harry24k/adversarial-attacks-pytorch\n",
    "import torchattacks\n",
    "print(\"torchattacks %s\"%(torchattacks.__version__))\n",
    "\n",
    "# https://github.com/bethgelab/foolbox\n",
    "import foolbox as fb\n",
    "print(\"foolbox %s\"%(fb.__version__))\n",
    "\n",
    "# https://github.com/IBM/adversarial-robustness-toolbox\n",
    "import art\n",
    "import art.attacks.evasion as evasion\n",
    "from art.classifiers import PyTorchClassifier\n",
    "print(\"art %s\"%(art.__version__))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.1. Linf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FGSM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: Wong2020Fast\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.48 (25 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.48 (18 ms)\n",
      "- ART\n",
      "- Robust Acc: 0.48 (67 ms)\n",
      "\n",
      "Model: Rice2020Overfitting\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.62 (91 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.62 (55 ms)\n",
      "- ART\n",
      "- Robust Acc: 0.62 (733 ms)\n",
      "\n",
      "Model: Carmon2019Unlabeled\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.68 (16 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.68 (26 ms)\n",
      "- ART\n",
      "- Robust Acc: 0.68 (217 ms)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for model_name in model_list:\n",
    "    print('Model: {}'.format(model_name))\n",
    "    model = load_model(model_name, norm='Linf').to(device)\n",
    "    \n",
    "    print(\"- Torchattacks\")\n",
    "    atk = torchattacks.FGSM(model, eps=8/255)\n",
    "    start = datetime.datetime.now()\n",
    "    adv_images = atk(images, labels)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    print('- Robust Acc: {} ({} ms)'.format(acc, int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print(\"- Foolbox\")\n",
    "    fmodel = fb.PyTorchModel(model, bounds=(0, 1))\n",
    "    atk = fb.attacks.LinfFastGradientAttack(random_start=False)\n",
    "    start = datetime.datetime.now()\n",
    "    _, adv_images, _ = atk(fmodel, images.to('cuda:0'), labels.to('cuda:0'), epsilons=8/255)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    print('- Robust Acc: {} ({} ms)'.format(acc, int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print(\"- ART\")\n",
    "    classifier = PyTorchClassifier(model=model, clip_values=(0, 1),\n",
    "                                   loss=nn.CrossEntropyLoss(),\n",
    "                                   optimizer=optim.Adam(model.parameters(), lr=0.01),\n",
    "                                   input_shape=(3, 32, 32), nb_classes=10)\n",
    "    atk = evasion.FastGradientMethod(norm=np.inf, batch_size=50,\n",
    "                                     classifier=classifier, eps=8/255)\n",
    "    start = datetime.datetime.now()\n",
    "    adv_images = torch.tensor(atk.generate(images, labels)).to(device)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    print('- Robust Acc: {} ({} ms)'.format(acc, int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BIM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: Wong2020Fast\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.44 (170 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.44 (230 ms)\n",
      "- ART\n",
      "- Robust Acc: 0.44 (485 ms)\n",
      "\n",
      "Model: Rice2020Overfitting\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.58 (2798 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.58 (3331 ms)\n",
      "- ART\n",
      "- Robust Acc: 0.58 (4695 ms)\n",
      "\n",
      "Model: Carmon2019Unlabeled\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.58 (732 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.58 (917 ms)\n",
      "- ART\n",
      "- Robust Acc: 0.58 (1414 ms)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for model_name in model_list:\n",
    "    print('Model: {}'.format(model_name))\n",
    "    model = load_model(model_name, norm='Linf').to(device)\n",
    "    \n",
    "    print(\"- Torchattacks\")\n",
    "    atk = torchattacks.BIM(model, eps=8/255, alpha=2/255, steps=10)\n",
    "    start = datetime.datetime.now()\n",
    "    adv_images = atk(images, labels)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    print('- Robust Acc: {} ({} ms)'.format(acc, int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print(\"- Foolbox\")\n",
    "    fmodel = fb.PyTorchModel(model, bounds=(0, 1))\n",
    "    atk = fb.attacks.LinfBasicIterativeAttack(abs_stepsize=2/255, steps=10, random_start=False)\n",
    "    start = datetime.datetime.now()\n",
    "    _, adv_images, _ = atk(fmodel, images.to('cuda:0'), labels.to('cuda:0'), epsilons=8/255)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    print('- Robust Acc: {} ({} ms)'.format(acc, int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print(\"- ART\")\n",
    "    classifier = PyTorchClassifier(model=model, clip_values=(0, 1),\n",
    "                                   loss=nn.CrossEntropyLoss(),\n",
    "                                   optimizer=optim.Adam(model.parameters(), lr=0.01),\n",
    "                                   input_shape=(3, 32, 32), nb_classes=10)\n",
    "    atk = evasion.BasicIterativeMethod(batch_size=50,\n",
    "                                       classifier=classifier, eps=8/255,\n",
    "                                       eps_step=2/255, max_iter=10)\n",
    "    start = datetime.datetime.now()\n",
    "    adv_images = torch.tensor(atk.generate(images, labels)).to(device)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    print('- Robust Acc: {} ({} ms)'.format(acc, int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PGD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: Wong2020Fast\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.44 (140 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.44 (237 ms)\n",
      "- ART\n",
      "- Robust Acc: 0.44 (480 ms)\n",
      "\n",
      "Model: Rice2020Overfitting\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.58 (2697 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.58 (3353 ms)\n",
      "- ART\n",
      "- Robust Acc: 0.58 (4720 ms)\n",
      "\n",
      "Model: Carmon2019Unlabeled\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.58 (682 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.58 (920 ms)\n",
      "- ART\n",
      "- Robust Acc: 0.58 (1426 ms)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for model_name in model_list:\n",
    "    print('Model: {}'.format(model_name))\n",
    "    model = load_model(model_name, norm='Linf').to(device)\n",
    "    \n",
    "    print(\"- Torchattacks\")\n",
    "    atk = torchattacks.PGD(model, eps=8/255, alpha=2/255, steps=10, random_start=False)\n",
    "    start = datetime.datetime.now()\n",
    "    adv_images = atk(images, labels)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    print('- Robust Acc: {} ({} ms)'.format(acc, int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print(\"- Foolbox\")\n",
    "    fmodel = fb.PyTorchModel(model, bounds=(0, 1))\n",
    "    atk = fb.attacks.LinfPGD(abs_stepsize=2/255, steps=10, random_start=False)\n",
    "    start = datetime.datetime.now()\n",
    "    _, adv_images, _ = atk(fmodel, images.to('cuda:0'), labels.to('cuda:0'), epsilons=8/255)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    print('- Robust Acc: {} ({} ms)'.format(acc, int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print(\"- ART\")\n",
    "    classifier = PyTorchClassifier(model=model, clip_values=(0, 1),\n",
    "                                   loss=nn.CrossEntropyLoss(),\n",
    "                                   optimizer=optim.Adam(model.parameters(), lr=0.01),\n",
    "                                   input_shape=(3, 32, 32), nb_classes=10)\n",
    "    atk = evasion.ProjectedGradientDescent(batch_size=50, num_random_init=0,\n",
    "                                           classifier=classifier, eps=8/255,\n",
    "                                           eps_step=2/255, max_iter=10)\n",
    "    start = datetime.datetime.now()\n",
    "    adv_images = torch.tensor(atk.generate(images, labels)).to(device)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    print('- Robust Acc: {} ({} ms)'.format(acc, int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.2. L2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DeepFool"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: Wong2020Fast\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.0 / L2: 1.0 (9952 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.4 / L2: 0.41 (1992 ms)\n",
      "- ART\n",
      "- Robust Acc: 0.04 / L2: 1.1 (18128 ms)\n",
      "\n",
      "Model: Rice2020Overfitting\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.0 / L2: 1.2 (39131 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.36 / L2: 0.47 (20359 ms)\n",
      "- ART\n",
      "- Robust Acc: 0.1 / L2: 1.3 (29016 ms)\n",
      "\n",
      "Model: Carmon2019Unlabeled\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.0 / L2: 1.8 (11555 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.46 / L2: 0.47 (5996 ms)\n",
      "- ART\n",
      "- Robust Acc: 0.06 / L2: 1.9 (8667 ms)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for model_name in model_list:\n",
    "    print('Model: {}'.format(model_name))\n",
    "    model = load_model(model_name, norm='Linf').to(device)\n",
    "    \n",
    "    print(\"- Torchattacks\")\n",
    "    atk = torchattacks.DeepFool(model, steps=50, overshoot=0.02)\n",
    "    start = datetime.datetime.now()\n",
    "    adv_images = atk(images, labels)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    l2 = l2_distance(model, images, adv_images, labels, device=device)\n",
    "    print('- Robust Acc: {} / L2: {:1.2} ({} ms)'.format(acc, l2,\n",
    "                                                         int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print(\"- Foolbox\")\n",
    "    fmodel = fb.PyTorchModel(model, bounds=(0, 1))\n",
    "    atk = fb.attacks.L2DeepFoolAttack(steps=50, candidates=10, overshoot=0.02)\n",
    "    start = datetime.datetime.now()\n",
    "    _, adv_images, _ = atk(fmodel, images.to('cuda'), labels.to('cuda'), epsilons=1)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    l2 = l2_distance(model, images, adv_images, labels, device=device)\n",
    "    print('- Robust Acc: {} / L2: {:1.2} ({} ms)'.format(acc, l2,\n",
    "                                                         int((end-start).total_seconds()*1000)))\n",
    "\n",
    "    print(\"- ART\")\n",
    "    classifier = PyTorchClassifier(model=model, clip_values=(0, 1),\n",
    "                                   loss=nn.CrossEntropyLoss(),\n",
    "                                   optimizer=optim.Adam(model.parameters(), lr=0.01),\n",
    "                                   input_shape=(3, 32, 32), nb_classes=10)\n",
    "    atk = evasion.DeepFool(classifier=classifier, max_iter=50,\n",
    "                           batch_size=50)\n",
    "\n",
    "    start = datetime.datetime.now()\n",
    "    adv_images = torch.tensor(atk.generate(images, labels)).to(device)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    l2 = l2_distance(model, images, adv_images, labels, device=device)\n",
    "    print('- Robust Acc: {} / L2: {:1.2} ({} ms)'.format(acc, l2,\n",
    "                                                         int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: Wong2020Fast\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.14 / L2: 0.61 (4372 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.32 / L2: 0.41 (4550 ms)\n",
      "- ART\n",
      "- Robust Acc: 0.24 / L2: 0.67 (72051 ms)\n",
      "\n",
      "Model: Rice2020Overfitting\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.22 / L2: 0.56 (43656 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.34 / L2: 0.43 (44207 ms)\n",
      "- ART\n",
      "- Robust Acc: 0.26 / L2: 0.65 (705790 ms)\n",
      "\n",
      "Model: Carmon2019Unlabeled\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.26 / L2: 0.48 (13358 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.32 / L2: 0.42 (13433 ms)\n",
      "- ART\n",
      "- Robust Acc: 0.26 / L2: 0.62 (206078 ms)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for model_name in model_list:\n",
    "    print('Model: {}'.format(model_name))\n",
    "    model = load_model(model_name, norm='Linf').to(device)\n",
    "    \n",
    "    print(\"- Torchattacks\")\n",
    "    atk = torchattacks.CW(model, c=1, kappa=0, steps=100, lr=0.01)\n",
    "    start = datetime.datetime.now()\n",
    "    adv_images = atk(images, labels)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    l2 = l2_distance(model, images, adv_images, labels, device=device)\n",
    "    print('- Robust Acc: {} / L2: {:1.2} ({} ms)'.format(acc, l2,\n",
    "                                                         int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print(\"- Foolbox\")\n",
    "    fmodel = fb.PyTorchModel(model, bounds=(0, 1))\n",
    "    atk = fb.attacks.L2CarliniWagnerAttack(binary_search_steps=1, initial_const=1,\n",
    "                                           confidence=0, steps=100, stepsize=0.01)\n",
    "    start = datetime.datetime.now()\n",
    "    _, adv_images, _ = atk(fmodel, images.to('cuda'), labels.to('cuda'), epsilons=1)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    l2 = l2_distance(model, images, adv_images, labels, device=device)\n",
    "    print('- Robust Acc: {} / L2: {:1.2} ({} ms)'.format(acc, l2,\n",
    "                                                         int((end-start).total_seconds()*1000)))\n",
    "\n",
    "    print(\"- ART\")\n",
    "    classifier = PyTorchClassifier(model=model, clip_values=(0, 1),\n",
    "                                   loss=nn.CrossEntropyLoss(),\n",
    "                                   optimizer=optim.Adam(model.parameters(), lr=0.01),\n",
    "                                   input_shape=(3, 32, 32), nb_classes=10)\n",
    "    atk = evasion.CarliniL2Method(batch_size=50, classifier=classifier, \n",
    "                                  binary_search_steps=1, initial_const=1,\n",
    "                                  confidence=0, max_iter=100,\n",
    "                                  learning_rate=0.01)\n",
    "    start = datetime.datetime.now()\n",
    "    adv_images = torch.tensor(atk.generate(images, labels)).to(device)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    l2 = l2_distance(model, images, adv_images, labels, device=device)\n",
    "    print('- Robust Acc: {} / L2: {:1.2} ({} ms)'.format(acc, l2,\n",
    "                                                         int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PGD L2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: Wong2020Fast\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.68 / L2: 0.5 (163 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.68 / L2: 0.5 (266 ms)\n",
      "- ART\n",
      "- Robust Acc: 0.68 / L2: 0.5 (479 ms)\n",
      "\n",
      "Model: Rice2020Overfitting\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.7 / L2: 0.5 (2801 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.7 / L2: 0.5 (3494 ms)\n",
      "- ART\n",
      "- Robust Acc: 0.7 / L2: 0.5 (4797 ms)\n",
      "\n",
      "Model: Carmon2019Unlabeled\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.68 / L2: 0.5 (719 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.68 / L2: 0.5 (959 ms)\n",
      "- ART\n",
      "- Robust Acc: 0.68 / L2: 0.5 (1424 ms)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for model_name in model_list:\n",
    "    print('Model: {}'.format(model_name))\n",
    "    model = load_model(model_name, norm='Linf').cuda()\n",
    "    \n",
    "    print(\"- Torchattacks\")\n",
    "    atk = torchattacks.PGDL2(model, eps=128/255, alpha=15/255, steps=10, random_start=False)\n",
    "    start = datetime.datetime.now()\n",
    "    adv_images = atk(images, labels)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    l2 = l2_distance(model, images, adv_images, labels, device=device)\n",
    "    print('- Robust Acc: {} / L2: {:1.2} ({} ms)'.format(acc, l2,\n",
    "                                                         int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print(\"- Foolbox\")\n",
    "    fmodel = fb.PyTorchModel(model, bounds=(0, 1))\n",
    "    atk = fb.attacks.L2PGD(abs_stepsize=15/255, steps=10, random_start=False)\n",
    "    start = datetime.datetime.now()\n",
    "    _, adv_images, _ = atk(fmodel, images.to('cuda:0'), labels.to('cuda:0'), epsilons=128/255)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    l2 = l2_distance(model, images, adv_images, labels, device=device)\n",
    "    print('- Robust Acc: {} / L2: {:1.2} ({} ms)'.format(acc, l2,\n",
    "                                                         int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print(\"- ART\")\n",
    "    classifier = PyTorchClassifier(model=model, clip_values=(0, 1),\n",
    "                                   loss=nn.CrossEntropyLoss(),\n",
    "                                   optimizer=optim.Adam(model.parameters(), lr=0.01),\n",
    "                                   input_shape=(3, 32, 32), nb_classes=10)\n",
    "    atk = evasion.ProjectedGradientDescent(batch_size=50, num_random_init=0,\n",
    "                                           norm = 2, classifier=classifier, eps=128/255,\n",
    "                                           eps_step=15/255, max_iter=10)\n",
    "    start = datetime.datetime.now()\n",
    "    adv_images = torch.tensor(atk.generate(images, labels)).to(device)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    l2 = l2_distance(model, images, adv_images, labels, device=device)\n",
    "    print('- Robust Acc: {} / L2: {:1.2} ({} ms)'.format(acc, l2,\n",
    "                                                         int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Torchattacks and AutoAttack\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchattacks.attack import Attack\n",
    "import autoattack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AutoAttack(Attack):\n",
    "    def __init__(self, model, eps, norm, seed, version):\n",
    "        super(AutoAttack, self).__init__(\"AutoAttack\", model)\n",
    "        self.adversary = autoattack.AutoAttack(self.model, norm=norm, seed=seed,\n",
    "                                               eps=eps, version=version, verbose=False)\n",
    "        self._attack_mode = 'only_default'\n",
    "\n",
    "    def forward(self, images, labels):\n",
    "        adv_images = self.adversary.run_standard_evaluation(images.cuda(), labels.cuda(),\n",
    "                                                            bs=images.shape[0])\n",
    "        return adv_images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_list = ['Wong2020Fast']\n",
    "versions = ['standard', 'plus', 'rand']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: Wong2020Fast\n",
      "- Torchattacks (standard)\n",
      "- Robust Acc: 0.4 (82904 ms)\n",
      "- AutoAttack (standard)\n",
      "- Robust Acc: 0.4 (103209 ms)\n",
      "- Torchattacks (plus)\n",
      "- Robust Acc: 0.4 (175182 ms)\n",
      "- AutoAttack (plus)\n",
      "- Robust Acc: 0.4 (199241 ms)\n",
      "- Torchattacks (rand)\n",
      "- Robust Acc: 0.44 (93462 ms)\n",
      "- AutoAttack (rand)\n",
      "- Robust Acc: 0.44 (93450 ms)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for model_name in model_list:\n",
    "    print('Model: {}'.format(model_name))\n",
    "    model = load_model(model_name, norm='Linf').to(device)\n",
    "    \n",
    "    for version in versions:\n",
    "        print(\"- Torchattacks (%s)\"%(version))\n",
    "        atk = torchattacks.AutoAttack(model, norm='Linf', eps=8/255, seed=5, version=version)\n",
    "        start = datetime.datetime.now()\n",
    "        adv_images = atk(images, labels)\n",
    "        end = datetime.datetime.now()\n",
    "        acc = clean_accuracy(model, adv_images, labels)\n",
    "        print('- Robust Acc: {} ({} ms)'.format(acc, int((end-start).total_seconds()*1000)))\n",
    "\n",
    "        print(\"- AutoAttack (%s)\"%(version))\n",
    "        atk = AutoAttack(model, norm='Linf', eps=8/255, seed=5, version=version)\n",
    "        start = datetime.datetime.now()\n",
    "        adv_images = atk(images, labels)\n",
    "        end = datetime.datetime.now()\n",
    "        acc = clean_accuracy(model, adv_images, labels)\n",
    "        print('- Robust Acc: {} ({} ms)'.format(acc, int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
